Problem statement

The objective is to classify images of dogs and cats accurately using deep learning techniques. The task involves training two models: a custom neural network and a fine-tuned pre-trained VGG16 model, to compare their effectiveness. The performance evaluation will assess key metrics such as accuracy, precision, recall, F1-score, and analyze specific cases where the models fail. Additionally, the work will focus on the clarity and interpretability of the findings through well-structured code and data exploration.

Importing Libaries

import tensorflow as tf

gpus = tf.config.experimental.list_physical_devices('GPU')
print("GPUs detected:", gpus)

# Attempt to set visible devices if GPUs are detected
if gpus:
    try:
        tf.config.experimental.set_visible_devices([], 'GPU')
    except RuntimeError as e:
        print(e) 

# Now proceed with other imports
from tensorflow import keras
from tensorflow.keras import layers
import pathlib
from tensorflow.keras.utils import image_dataset_from_directory
import matplotlib.pyplot as plt
import os
import random
from tensorflow.keras.preprocessing.image import load_img
import numpy as np
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Conv2D, MaxPooling2D, Flatten, Dense, Dropout
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from tensorflow.keras.preprocessing.image import img_to_array
from tensorflow.keras import optimizers

GPUs detected: [PhysicalDevice(name='/physical_device:GPU:0', device_type='GPU')]

Obtaining the dataset

data_folder = pathlib.Path('../CSCN8010/data/kaggle_dogs_vs_cats_small')

train_dataset = image_dataset_from_directory(
    data_folder / "train",
    image_size=(180, 180),
    batch_size=32)
validation_dataset = image_dataset_from_directory(
    data_folder / "validation",
    image_size=(180, 180),
    batch_size=32)
test_dataset = image_dataset_from_directory(
    data_folder / "test",
    image_size=(180, 180),
    batch_size=32)

Found 2000 files belonging to 2 classes.
Found 1000 files belonging to 2 classes.
Found 2000 files belonging to 2 classes.

EDA

import os

train_dir = data_folder / "train"

print("Training data:")
for label in os.listdir(train_dir):
    label_path = os.path.join(train_dir, label)
    if os.path.isdir(label_path): 
        print(f"{label}: {len(os.listdir(label_path))} images")

print("Training data:")
for label in os.listdir(train_dir):
    label_path = os.path.join(train_dir, label)
    if os.path.isdir(label_path): 
        print(f"{label}: {len(os.listdir(label_path))} images")

Training data:
cat: 1000 images
dog: 1000 images
Training data:
cat: 1000 images
dog: 1000 images

train_images_dir = os.path.join(data_folder, 'train')  

def plot_sample_images_flat(folder, label, n=5):
    class_folder = os.path.join(folder, label)
    
    # List image files in the subdirectory
    image_files = os.listdir(class_folder)
    
    if len(image_files) < n:
        n = len(image_files)
    
    selected_files = random.sample(image_files, n)
    
    plt.figure(figsize=(15, 5))
    for i, img_file in enumerate(selected_files):
        img_path = os.path.join(class_folder, img_file)  
        img = load_img(img_path, target_size=(128, 128))  
        plt.subplot(1, n, i + 1)
        plt.imshow(img)
        plt.title(label)
        plt.axis('off')
    plt.show()

plot_sample_images_flat(train_images_dir, 'cat')
plot_sample_images_flat(train_images_dir, 'dog')

dimensions = []
for subdir in ['cat', 'dog']: 
    subdir_path = os.path.join(train_images_dir, subdir)
    for filename in os.listdir(subdir_path):
        if filename.endswith('.jpg'):
            img_path = os.path.join(subdir_path, filename)
            img = img_to_array(load_img(img_path)) 
            dimensions.append(img.shape[:2]) 

dimensions = np.array(dimensions)

plt.figure(figsize=(10, 5))
plt.hist(dimensions[:, 0], bins=20, alpha=0.7, label='Height')  
plt.hist(dimensions[:, 1], bins=20, alpha=0.7, label='Width') 
plt.title("Image Dimension Distribution")
plt.xlabel("Pixels")
plt.ylabel("Frequency")
plt.legend()
plt.show()

The graph above shows that most of the images are consistent in size between 400 -600 pixels for both width and height.

Defining the CNN Model

inputs = keras.Input(shape=(180, 180, 3))
x = layers.Rescaling(1./255)(inputs)
x = layers.Conv2D(filters=32, kernel_size=3, activation="relu")(x)
x = layers.MaxPooling2D(pool_size=2)(x)
x = layers.Conv2D(filters=64, kernel_size=3, activation="relu")(x)
x = layers.MaxPooling2D(pool_size=2)(x)
x = layers.Conv2D(filters=128, kernel_size=3, activation="relu")(x)
x = layers.MaxPooling2D(pool_size=2)(x)
x = layers.Conv2D(filters=256, kernel_size=3, activation="relu")(x)
x = layers.MaxPooling2D(pool_size=2)(x)
x = layers.Conv2D(filters=256, kernel_size=3, activation="relu")(x)
x = layers.Flatten()(x)
outputs = layers.Dense(1, activation="sigmoid")(x)
model = keras.Model(inputs=inputs, outputs=outputs)

# Compile the model

model.compile(loss="binary_crossentropy",
              optimizer="rmsprop",
              metrics=["accuracy"])

model.summary()

Model: "model"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 input_1 (InputLayer)        [(None, 180, 180, 3)]     0         
                                                                 
 rescaling (Rescaling)       (None, 180, 180, 3)       0         
                                                                 
 conv2d (Conv2D)             (None, 178, 178, 32)      896       
                                                                 
 max_pooling2d (MaxPooling2  (None, 89, 89, 32)        0         
 D)                                                              
                                                                 
 conv2d_1 (Conv2D)           (None, 87, 87, 64)        18496     
                                                                 
 max_pooling2d_1 (MaxPoolin  (None, 43, 43, 64)        0         
 g2D)                                                            
                                                                 
 conv2d_2 (Conv2D)           (None, 41, 41, 128)       73856     
                                                                 
 max_pooling2d_2 (MaxPoolin  (None, 20, 20, 128)       0         
 g2D)                                                            
                                                                 
 conv2d_3 (Conv2D)           (None, 18, 18, 256)       295168    
                                                                 
 max_pooling2d_3 (MaxPoolin  (None, 9, 9, 256)         0         
 g2D)                                                            
                                                                 
 conv2d_4 (Conv2D)           (None, 7, 7, 256)         590080    
                                                                 
 flatten (Flatten)           (None, 12544)             0         
                                                                 
 dense (Dense)               (None, 1)                 12545     
                                                                 
=================================================================
Total params: 991041 (3.78 MB)
Trainable params: 991041 (3.78 MB)
Non-trainable params: 0 (0.00 Byte)
_________________________________________________________________

import os
import numpy as np
from tensorflow.keras.callbacks import ModelCheckpoint
from tensorflow.keras.models import load_model

current_model_dir = "./models_v2"
if not os.path.exists(current_model_dir):
    os.makedirs(current_model_dir)

# Callback to save all model versions in the unique directory
callbacks = [
    keras.callbacks.ModelCheckpoint(
        filepath=f"{current_model_dir}/feature_extraction_epoch_{{epoch:02d}}.keras",
        save_best_only=False,
        verbose=1
    )
]


# Train the model
history = model.fit(
    train_dataset,                     
    epochs=30,                        
    validation_data=validation_dataset,  
    callbacks=callbacks 
)

# Extract validation losses from history
val_losses = history.history['val_loss']

# Find the top 3 epochs with the lowest validation loss
best_epochs = np.argsort(val_losses)[:3]  
print(f"Top 3 Epochs with Lowest Validation Loss: {best_epochs + 1}")  

# Evaluate the top 3 models
for epoch in best_epochs:
    model_path = f"./models/model_epoch_{epoch+1:02d}.h5"  
    print(f"\nEvaluating model from epoch {epoch + 1}: {model_path}")
    
    best_model = load_model(model_path)
    
    test_loss, test_accuracy = best_model.evaluate(test_dataset)
    print(f"Test Loss: {test_loss}, Test Accuracy: {test_accuracy}")

# Save the best model (overall lowest validation loss)
best_epoch = best_epochs[0]  
best_model_path = f"./models/model_epoch_{best_epoch+1:02d}.h5"  

# Load the best model
best_model = load_model(best_model_path)

# Save the best model to a final file
best_model.save('./models/best_model_overall.h5')
print("\nBest model saved as './models/best_model_overall.h5'")

Epoch 1/30
63/63 [==============================] - ETA: 0s - loss: 0.7782 - accuracy: 0.5080
Epoch 1: saving model to ./models_v2/feature_extraction_epoch_01.keras
63/63 [==============================] - 33s 504ms/step - loss: 0.7782 - accuracy: 0.5080 - val_loss: 0.6919 - val_accuracy: 0.5290
Epoch 2/30
63/63 [==============================] - ETA: 0s - loss: 0.6969 - accuracy: 0.5270
Epoch 2: saving model to ./models_v2/feature_extraction_epoch_02.keras
63/63 [==============================] - 30s 482ms/step - loss: 0.6969 - accuracy: 0.5270 - val_loss: 0.6762 - val_accuracy: 0.6140
Epoch 3/30
63/63 [==============================] - ETA: 0s - loss: 0.6993 - accuracy: 0.5935
Epoch 3: saving model to ./models_v2/feature_extraction_epoch_03.keras
63/63 [==============================] - 30s 480ms/step - loss: 0.6993 - accuracy: 0.5935 - val_loss: 0.8372 - val_accuracy: 0.5050
Epoch 4/30
63/63 [==============================] - ETA: 0s - loss: 0.6990 - accuracy: 0.6350
Epoch 4: saving model to ./models_v2/feature_extraction_epoch_04.keras
63/63 [==============================] - 28s 451ms/step - loss: 0.6990 - accuracy: 0.6350 - val_loss: 0.6224 - val_accuracy: 0.6600
Epoch 5/30
63/63 [==============================] - ETA: 0s - loss: 0.6111 - accuracy: 0.7030
Epoch 5: saving model to ./models_v2/feature_extraction_epoch_05.keras
63/63 [==============================] - 28s 447ms/step - loss: 0.6111 - accuracy: 0.7030 - val_loss: 0.6504 - val_accuracy: 0.6080
Epoch 6/30
63/63 [==============================] - ETA: 0s - loss: 0.5821 - accuracy: 0.6925
Epoch 6: saving model to ./models_v2/feature_extraction_epoch_06.keras
63/63 [==============================] - 28s 451ms/step - loss: 0.5821 - accuracy: 0.6925 - val_loss: 0.7471 - val_accuracy: 0.6410
Epoch 7/30
63/63 [==============================] - ETA: 0s - loss: 0.5594 - accuracy: 0.7270
Epoch 7: saving model to ./models_v2/feature_extraction_epoch_07.keras
63/63 [==============================] - 28s 447ms/step - loss: 0.5594 - accuracy: 0.7270 - val_loss: 0.5739 - val_accuracy: 0.7040
Epoch 8/30
63/63 [==============================] - ETA: 0s - loss: 0.5091 - accuracy: 0.7460
Epoch 8: saving model to ./models_v2/feature_extraction_epoch_08.keras
63/63 [==============================] - 28s 451ms/step - loss: 0.5091 - accuracy: 0.7460 - val_loss: 0.5478 - val_accuracy: 0.7300
Epoch 9/30
63/63 [==============================] - ETA: 0s - loss: 0.4786 - accuracy: 0.7675
Epoch 9: saving model to ./models_v2/feature_extraction_epoch_09.keras
63/63 [==============================] - 29s 452ms/step - loss: 0.4786 - accuracy: 0.7675 - val_loss: 0.5757 - val_accuracy: 0.7200
Epoch 10/30
63/63 [==============================] - ETA: 0s - loss: 0.4269 - accuracy: 0.8010
Epoch 10: saving model to ./models_v2/feature_extraction_epoch_10.keras
63/63 [==============================] - 28s 450ms/step - loss: 0.4269 - accuracy: 0.8010 - val_loss: 0.5392 - val_accuracy: 0.7240
Epoch 11/30
63/63 [==============================] - ETA: 0s - loss: 0.3896 - accuracy: 0.8170
Epoch 11: saving model to ./models_v2/feature_extraction_epoch_11.keras
63/63 [==============================] - 29s 452ms/step - loss: 0.3896 - accuracy: 0.8170 - val_loss: 0.6594 - val_accuracy: 0.7140
Epoch 12/30
63/63 [==============================] - ETA: 0s - loss: 0.3584 - accuracy: 0.8425
Epoch 12: saving model to ./models_v2/feature_extraction_epoch_12.keras
63/63 [==============================] - 29s 453ms/step - loss: 0.3584 - accuracy: 0.8425 - val_loss: 0.6795 - val_accuracy: 0.6930
Epoch 13/30
63/63 [==============================] - ETA: 0s - loss: 0.3019 - accuracy: 0.8730
Epoch 13: saving model to ./models_v2/feature_extraction_epoch_13.keras
63/63 [==============================] - 28s 451ms/step - loss: 0.3019 - accuracy: 0.8730 - val_loss: 0.7136 - val_accuracy: 0.7180
Epoch 14/30
63/63 [==============================] - ETA: 0s - loss: 0.2399 - accuracy: 0.8945
Epoch 14: saving model to ./models_v2/feature_extraction_epoch_14.keras
63/63 [==============================] - 29s 453ms/step - loss: 0.2399 - accuracy: 0.8945 - val_loss: 0.7234 - val_accuracy: 0.7220
Epoch 15/30
63/63 [==============================] - ETA: 0s - loss: 0.2038 - accuracy: 0.9195
Epoch 15: saving model to ./models_v2/feature_extraction_epoch_15.keras
63/63 [==============================] - 29s 462ms/step - loss: 0.2038 - accuracy: 0.9195 - val_loss: 0.8531 - val_accuracy: 0.7120
Epoch 16/30
63/63 [==============================] - ETA: 0s - loss: 0.1662 - accuracy: 0.9355
Epoch 16: saving model to ./models_v2/feature_extraction_epoch_16.keras
63/63 [==============================] - 29s 463ms/step - loss: 0.1662 - accuracy: 0.9355 - val_loss: 0.9938 - val_accuracy: 0.7050
Epoch 17/30
63/63 [==============================] - ETA: 0s - loss: 0.1254 - accuracy: 0.9525
Epoch 17: saving model to ./models_v2/feature_extraction_epoch_17.keras
63/63 [==============================] - 30s 481ms/step - loss: 0.1254 - accuracy: 0.9525 - val_loss: 0.9559 - val_accuracy: 0.7400
Epoch 18/30
63/63 [==============================] - ETA: 0s - loss: 0.1121 - accuracy: 0.9635
Epoch 18: saving model to ./models_v2/feature_extraction_epoch_18.keras
63/63 [==============================] - 30s 479ms/step - loss: 0.1121 - accuracy: 0.9635 - val_loss: 1.1621 - val_accuracy: 0.7430
Epoch 19/30
63/63 [==============================] - ETA: 0s - loss: 0.1236 - accuracy: 0.9550
Epoch 19: saving model to ./models_v2/feature_extraction_epoch_19.keras
63/63 [==============================] - 31s 487ms/step - loss: 0.1236 - accuracy: 0.9550 - val_loss: 1.4168 - val_accuracy: 0.6770
Epoch 20/30
63/63 [==============================] - ETA: 0s - loss: 0.0705 - accuracy: 0.9795
Epoch 20: saving model to ./models_v2/feature_extraction_epoch_20.keras
63/63 [==============================] - 30s 467ms/step - loss: 0.0705 - accuracy: 0.9795 - val_loss: 1.4589 - val_accuracy: 0.7370
Epoch 21/30
63/63 [==============================] - ETA: 0s - loss: 0.0780 - accuracy: 0.9780
Epoch 21: saving model to ./models_v2/feature_extraction_epoch_21.keras
63/63 [==============================] - 30s 477ms/step - loss: 0.0780 - accuracy: 0.9780 - val_loss: 1.7510 - val_accuracy: 0.7280
Epoch 22/30
63/63 [==============================] - ETA: 0s - loss: 0.0899 - accuracy: 0.9745
Epoch 22: saving model to ./models_v2/feature_extraction_epoch_22.keras
63/63 [==============================] - 29s 466ms/step - loss: 0.0899 - accuracy: 0.9745 - val_loss: 1.6014 - val_accuracy: 0.7260
Epoch 23/30
63/63 [==============================] - ETA: 0s - loss: 0.0587 - accuracy: 0.9825
Epoch 23: saving model to ./models_v2/feature_extraction_epoch_23.keras
63/63 [==============================] - 29s 466ms/step - loss: 0.0587 - accuracy: 0.9825 - val_loss: 2.0013 - val_accuracy: 0.6900
Epoch 24/30
63/63 [==============================] - ETA: 0s - loss: 0.0672 - accuracy: 0.9770
Epoch 24: saving model to ./models_v2/feature_extraction_epoch_24.keras
63/63 [==============================] - 29s 467ms/step - loss: 0.0672 - accuracy: 0.9770 - val_loss: 1.5655 - val_accuracy: 0.7400
Epoch 25/30
63/63 [==============================] - ETA: 0s - loss: 0.0716 - accuracy: 0.9785
Epoch 25: saving model to ./models_v2/feature_extraction_epoch_25.keras
63/63 [==============================] - 30s 472ms/step - loss: 0.0716 - accuracy: 0.9785 - val_loss: 1.8810 - val_accuracy: 0.7310
Epoch 26/30
63/63 [==============================] - ETA: 0s - loss: 0.0507 - accuracy: 0.9845
Epoch 26: saving model to ./models_v2/feature_extraction_epoch_26.keras
63/63 [==============================] - 30s 472ms/step - loss: 0.0507 - accuracy: 0.9845 - val_loss: 2.2341 - val_accuracy: 0.7250
Epoch 27/30
63/63 [==============================] - ETA: 0s - loss: 0.0778 - accuracy: 0.9820
Epoch 27: saving model to ./models_v2/feature_extraction_epoch_27.keras
63/63 [==============================] - 29s 465ms/step - loss: 0.0778 - accuracy: 0.9820 - val_loss: 1.7260 - val_accuracy: 0.7610
Epoch 28/30
63/63 [==============================] - ETA: 0s - loss: 0.0512 - accuracy: 0.9870
Epoch 28: saving model to ./models_v2/feature_extraction_epoch_28.keras
63/63 [==============================] - 29s 462ms/step - loss: 0.0512 - accuracy: 0.9870 - val_loss: 1.8228 - val_accuracy: 0.7430
Epoch 29/30
63/63 [==============================] - ETA: 0s - loss: 0.0783 - accuracy: 0.9775
Epoch 29: saving model to ./models_v2/feature_extraction_epoch_29.keras
63/63 [==============================] - 29s 460ms/step - loss: 0.0783 - accuracy: 0.9775 - val_loss: 2.0788 - val_accuracy: 0.7350
Epoch 30/30
63/63 [==============================] - ETA: 0s - loss: 0.0585 - accuracy: 0.9815
Epoch 30: saving model to ./models_v2/feature_extraction_epoch_30.keras
63/63 [==============================] - 29s 452ms/step - loss: 0.0585 - accuracy: 0.9815 - val_loss: 1.9940 - val_accuracy: 0.7350
Top 3 Epochs with Lowest Validation Loss: [10  8  7]

Evaluating model from epoch 10: ./models/model_epoch_10.h5
63/63 [==============================] - 8s 117ms/step - loss: 0.6163 - accuracy: 0.6610
Test Loss: 0.6163120269775391, Test Accuracy: 0.6610000133514404

Evaluating model from epoch 8: ./models/model_epoch_08.h5
63/63 [==============================] - 7s 115ms/step - loss: 0.6705 - accuracy: 0.5895
Test Loss: 0.6704769730567932, Test Accuracy: 0.5895000100135803

Evaluating model from epoch 7: ./models/model_epoch_07.h5
63/63 [==============================] - 7s 114ms/step - loss: 0.6415 - accuracy: 0.6410
Test Loss: 0.6414811611175537, Test Accuracy: 0.640999972820282

Best model saved as './models/best_model_overall.h5'

/Users/elder/miniconda/envs/tensorflow_metal/lib/python3.10/site-packages/keras/src/engine/training.py:3079: UserWarning: You are saving your model as an HDF5 file via `model.save()`. This file format is considered legacy. We recommend using instead the native Keras format, e.g. `model.save('my_model.keras')`.
  saving_api.save_model(

import matplotlib.pyplot as plt

# Function to plot training history
def plot_training_history(history):
    # Extract metrics from the history object
    accuracy = history.history['accuracy']
    val_accuracy = history.history['val_accuracy']
    loss = history.history['loss']
    val_loss = history.history['val_loss']

    # Plot Accuracy
    plt.figure(figsize=(12, 5))
    plt.subplot(1, 2, 1)
    plt.plot(accuracy, label='Training Accuracy')
    plt.plot(val_accuracy, label='Validation Accuracy')
    plt.title('Training and Validation Accuracy')
    plt.xlabel('Epochs')
    plt.ylabel('Accuracy')
    plt.legend()

    # Plot Loss
    plt.subplot(1, 2, 2)
    plt.plot(loss, label='Training Loss')
    plt.plot(val_loss, label='Validation Loss')
    plt.title('Training and Validation Loss')
    plt.xlabel('Epochs')
    plt.ylabel('Loss')
    plt.legend()

    plt.show()

# Call the function to plot curves
plot_training_history(history)

Overfitting likely starts around epoch 15, when the validation loss diverges from the training loss, and the validation accuracy stops improving significantly.

# Evaluate the model on the test set
test_loss, test_accuracy = best_model.evaluate(test_dataset)

# Print results
print(f"Test Loss: {test_loss}")
print(f"Test Accuracy: {test_accuracy}")

63/63 [==============================] - 7s 114ms/step - loss: 0.6163 - accuracy: 0.6610
Test Loss: 0.6163119077682495
Test Accuracy: 0.6610000133514404

The first bespoke CNN model performed moderately well in classifying the Dogs vs. Cats dataset, with a test accuracy of 66.1% with a loss of 0.6163. The comparatively large loss indicates that there is potential for improvement, either through improved architecture, data augmentation, or regularisation to combat overfitting, even though it captures fundamental properties.

Augmentation

from tensorflow.keras import layers, Sequential

# Data Augmentation Layer
data_augmentation = Sequential(
    [
        layers.RandomFlip("horizontal"),
        layers.RandomRotation(0.1),
        layers.RandomZoom(0.2),
    ]
)

inputs = keras.Input(shape=(180, 180, 3))
x = data_augmentation(inputs)
x = layers.Rescaling(1./255)(x)
x = layers.Conv2D(filters=32, kernel_size=3, activation="relu")(x)
x = layers.MaxPooling2D(pool_size=2)(x)
x = layers.Conv2D(filters=64, kernel_size=3, activation="relu")(x)
x = layers.MaxPooling2D(pool_size=2)(x)
x = layers.Conv2D(filters=128, kernel_size=3, activation="relu")(x)
x = layers.MaxPooling2D(pool_size=2)(x)
x = layers.Conv2D(filters=256, kernel_size=3, activation="relu")(x)
x = layers.MaxPooling2D(pool_size=2)(x)
x = layers.Conv2D(filters=256, kernel_size=3, activation="relu")(x)
x = layers.Flatten()(x)
x = layers.Dropout(0.5)(x)
outputs = layers.Dense(1, activation="sigmoid")(x)
model = keras.Model(inputs=inputs, outputs=outputs)

model.compile(loss="binary_crossentropy",
              optimizer="rmsprop",
              metrics=["accuracy"])

model.summary()

callbacks = [
    keras.callbacks.ModelCheckpoint(
        filepath="./models/convnet_from_scratch_with_augmentation.h5",
        save_best_only=True,
        monitor="val_loss"
    )
]

# Train the model
history = model.fit(
    train_dataset,              # Training dataset
    validation_data=validation_dataset,  # Validation dataset
    epochs=100,                  # Number of epochs
    callbacks=callbacks         # Include callbacks
)

Model: "model_1"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 input_2 (InputLayer)        [(None, 180, 180, 3)]     0         
                                                                 
 sequential (Sequential)     (None, 180, 180, 3)       0         
                                                                 
 rescaling_1 (Rescaling)     (None, 180, 180, 3)       0         
                                                                 
 conv2d_5 (Conv2D)           (None, 178, 178, 32)      896       
                                                                 
 max_pooling2d_4 (MaxPoolin  (None, 89, 89, 32)        0         
 g2D)                                                            
                                                                 
 conv2d_6 (Conv2D)           (None, 87, 87, 64)        18496     
                                                                 
 max_pooling2d_5 (MaxPoolin  (None, 43, 43, 64)        0         
 g2D)                                                            
                                                                 
 conv2d_7 (Conv2D)           (None, 41, 41, 128)       73856     
                                                                 
 max_pooling2d_6 (MaxPoolin  (None, 20, 20, 128)       0         
 g2D)                                                            
                                                                 
 conv2d_8 (Conv2D)           (None, 18, 18, 256)       295168    
                                                                 
 max_pooling2d_7 (MaxPoolin  (None, 9, 9, 256)         0         
 g2D)                                                            
                                                                 
 conv2d_9 (Conv2D)           (None, 7, 7, 256)         590080    
                                                                 
 flatten_1 (Flatten)         (None, 12544)             0         
                                                                 
 dropout (Dropout)           (None, 12544)             0         
                                                                 
 dense_1 (Dense)             (None, 1)                 12545     
                                                                 
=================================================================
Total params: 991041 (3.78 MB)
Trainable params: 991041 (3.78 MB)
Non-trainable params: 0 (0.00 Byte)
_________________________________________________________________
Epoch 1/100
63/63 [==============================] - 30s 465ms/step - loss: 0.7236 - accuracy: 0.5185 - val_loss: 0.6879 - val_accuracy: 0.5000
Epoch 2/100
63/63 [==============================] - 29s 463ms/step - loss: 0.6921 - accuracy: 0.5505 - val_loss: 0.7192 - val_accuracy: 0.5080
Epoch 3/100
63/63 [==============================] - 29s 453ms/step - loss: 0.6906 - accuracy: 0.6030 - val_loss: 0.7545 - val_accuracy: 0.5470
Epoch 4/100
63/63 [==============================] - 29s 455ms/step - loss: 0.6526 - accuracy: 0.6455 - val_loss: 0.6360 - val_accuracy: 0.6390
Epoch 5/100
63/63 [==============================] - 29s 457ms/step - loss: 0.6250 - accuracy: 0.6640 - val_loss: 0.5781 - val_accuracy: 0.6970
Epoch 6/100
63/63 [==============================] - 29s 457ms/step - loss: 0.6152 - accuracy: 0.6665 - val_loss: 0.6415 - val_accuracy: 0.6150
Epoch 7/100
63/63 [==============================] - 29s 464ms/step - loss: 0.5962 - accuracy: 0.6885 - val_loss: 0.5762 - val_accuracy: 0.7020
Epoch 8/100
63/63 [==============================] - 29s 462ms/step - loss: 0.5948 - accuracy: 0.6920 - val_loss: 0.6133 - val_accuracy: 0.6620
Epoch 9/100
63/63 [==============================] - 29s 459ms/step - loss: 0.5692 - accuracy: 0.7125 - val_loss: 0.9701 - val_accuracy: 0.6120
Epoch 10/100
63/63 [==============================] - 29s 461ms/step - loss: 0.5671 - accuracy: 0.7065 - val_loss: 0.5612 - val_accuracy: 0.7180
Epoch 11/100
63/63 [==============================] - 29s 463ms/step - loss: 0.5475 - accuracy: 0.7245 - val_loss: 0.5493 - val_accuracy: 0.7130
Epoch 12/100
63/63 [==============================] - 30s 472ms/step - loss: 0.5419 - accuracy: 0.7280 - val_loss: 0.5502 - val_accuracy: 0.7160
Epoch 13/100
63/63 [==============================] - 29s 464ms/step - loss: 0.5307 - accuracy: 0.7345 - val_loss: 0.5157 - val_accuracy: 0.7440
Epoch 14/100
63/63 [==============================] - 30s 469ms/step - loss: 0.5258 - accuracy: 0.7445 - val_loss: 0.5539 - val_accuracy: 0.7320
Epoch 15/100
63/63 [==============================] - 30s 471ms/step - loss: 0.4976 - accuracy: 0.7595 - val_loss: 0.5929 - val_accuracy: 0.7100
Epoch 16/100
63/63 [==============================] - 29s 465ms/step - loss: 0.5052 - accuracy: 0.7540 - val_loss: 0.7539 - val_accuracy: 0.7030
Epoch 17/100
63/63 [==============================] - 30s 482ms/step - loss: 0.4856 - accuracy: 0.7775 - val_loss: 0.4832 - val_accuracy: 0.7700
Epoch 18/100
63/63 [==============================] - 32s 507ms/step - loss: 0.4803 - accuracy: 0.7745 - val_loss: 0.5627 - val_accuracy: 0.7540
Epoch 19/100
63/63 [==============================] - 30s 481ms/step - loss: 0.4833 - accuracy: 0.7765 - val_loss: 0.5657 - val_accuracy: 0.6940
Epoch 20/100
63/63 [==============================] - 29s 466ms/step - loss: 0.4623 - accuracy: 0.7865 - val_loss: 0.4790 - val_accuracy: 0.7900
Epoch 21/100
63/63 [==============================] - 29s 462ms/step - loss: 0.4351 - accuracy: 0.8075 - val_loss: 0.7629 - val_accuracy: 0.6920
Epoch 22/100
63/63 [==============================] - 30s 472ms/step - loss: 0.4244 - accuracy: 0.8160 - val_loss: 0.5366 - val_accuracy: 0.7640
Epoch 23/100
63/63 [==============================] - 29s 460ms/step - loss: 0.4386 - accuracy: 0.8025 - val_loss: 0.4376 - val_accuracy: 0.8230
Epoch 24/100
63/63 [==============================] - 29s 460ms/step - loss: 0.4287 - accuracy: 0.8030 - val_loss: 0.5266 - val_accuracy: 0.7830
Epoch 25/100
63/63 [==============================] - 30s 473ms/step - loss: 0.4207 - accuracy: 0.8085 - val_loss: 0.4863 - val_accuracy: 0.7910
Epoch 26/100
63/63 [==============================] - 30s 468ms/step - loss: 0.4018 - accuracy: 0.8130 - val_loss: 0.4207 - val_accuracy: 0.8110
Epoch 27/100
63/63 [==============================] - 29s 467ms/step - loss: 0.4113 - accuracy: 0.8090 - val_loss: 0.4293 - val_accuracy: 0.8290
Epoch 28/100
63/63 [==============================] - 30s 470ms/step - loss: 0.3695 - accuracy: 0.8455 - val_loss: 0.6394 - val_accuracy: 0.7410
Epoch 29/100
63/63 [==============================] - 29s 466ms/step - loss: 0.3914 - accuracy: 0.8275 - val_loss: 0.5115 - val_accuracy: 0.7560
Epoch 30/100
63/63 [==============================] - 30s 472ms/step - loss: 0.3877 - accuracy: 0.8245 - val_loss: 0.4234 - val_accuracy: 0.8130
Epoch 31/100
63/63 [==============================] - 29s 459ms/step - loss: 0.3794 - accuracy: 0.8395 - val_loss: 0.5173 - val_accuracy: 0.7720
Epoch 32/100
63/63 [==============================] - 29s 464ms/step - loss: 0.3537 - accuracy: 0.8470 - val_loss: 0.4594 - val_accuracy: 0.8110
Epoch 33/100
63/63 [==============================] - 29s 453ms/step - loss: 0.3716 - accuracy: 0.8340 - val_loss: 0.5484 - val_accuracy: 0.7920
Epoch 34/100
63/63 [==============================] - 30s 470ms/step - loss: 0.3576 - accuracy: 0.8420 - val_loss: 0.6936 - val_accuracy: 0.7860
Epoch 35/100
63/63 [==============================] - 29s 455ms/step - loss: 0.3496 - accuracy: 0.8530 - val_loss: 0.4480 - val_accuracy: 0.7960
Epoch 36/100
63/63 [==============================] - 29s 461ms/step - loss: 0.3377 - accuracy: 0.8470 - val_loss: 0.4604 - val_accuracy: 0.8260
Epoch 37/100
63/63 [==============================] - 29s 455ms/step - loss: 0.3397 - accuracy: 0.8595 - val_loss: 0.4819 - val_accuracy: 0.8230
Epoch 38/100
63/63 [==============================] - 29s 460ms/step - loss: 0.3149 - accuracy: 0.8665 - val_loss: 0.5601 - val_accuracy: 0.7790
Epoch 39/100
63/63 [==============================] - 29s 463ms/step - loss: 0.3346 - accuracy: 0.8580 - val_loss: 0.4799 - val_accuracy: 0.8270
Epoch 40/100
63/63 [==============================] - 30s 483ms/step - loss: 0.3275 - accuracy: 0.8600 - val_loss: 0.7472 - val_accuracy: 0.7340
Epoch 41/100
63/63 [==============================] - 29s 463ms/step - loss: 0.3160 - accuracy: 0.8610 - val_loss: 0.5182 - val_accuracy: 0.7790
Epoch 42/100
63/63 [==============================] - 30s 471ms/step - loss: 0.3090 - accuracy: 0.8665 - val_loss: 0.5387 - val_accuracy: 0.7980
Epoch 43/100
63/63 [==============================] - 30s 468ms/step - loss: 0.3070 - accuracy: 0.8630 - val_loss: 0.7566 - val_accuracy: 0.7500
Epoch 44/100
63/63 [==============================] - 30s 474ms/step - loss: 0.3100 - accuracy: 0.8645 - val_loss: 0.6193 - val_accuracy: 0.8060
Epoch 45/100
63/63 [==============================] - 30s 471ms/step - loss: 0.2816 - accuracy: 0.8765 - val_loss: 0.5567 - val_accuracy: 0.7730
Epoch 46/100
63/63 [==============================] - 29s 463ms/step - loss: 0.2899 - accuracy: 0.8845 - val_loss: 0.4383 - val_accuracy: 0.8410
Epoch 47/100
63/63 [==============================] - 30s 475ms/step - loss: 0.2591 - accuracy: 0.8870 - val_loss: 0.4563 - val_accuracy: 0.8450
Epoch 48/100
63/63 [==============================] - 30s 471ms/step - loss: 0.2662 - accuracy: 0.8895 - val_loss: 0.9870 - val_accuracy: 0.7710
Epoch 49/100
63/63 [==============================] - 30s 482ms/step - loss: 0.2816 - accuracy: 0.8870 - val_loss: 0.6514 - val_accuracy: 0.8050
Epoch 50/100
63/63 [==============================] - 30s 468ms/step - loss: 0.2706 - accuracy: 0.8840 - val_loss: 0.5273 - val_accuracy: 0.8280
Epoch 51/100
63/63 [==============================] - 30s 466ms/step - loss: 0.2667 - accuracy: 0.8880 - val_loss: 0.5044 - val_accuracy: 0.8310
Epoch 52/100
63/63 [==============================] - 30s 468ms/step - loss: 0.2511 - accuracy: 0.8980 - val_loss: 0.5847 - val_accuracy: 0.8180
Epoch 53/100
63/63 [==============================] - 29s 465ms/step - loss: 0.2583 - accuracy: 0.8980 - val_loss: 0.4742 - val_accuracy: 0.8220
Epoch 54/100
63/63 [==============================] - 29s 461ms/step - loss: 0.2554 - accuracy: 0.8965 - val_loss: 0.7118 - val_accuracy: 0.7910
Epoch 55/100
63/63 [==============================] - 29s 462ms/step - loss: 0.2685 - accuracy: 0.8885 - val_loss: 0.4811 - val_accuracy: 0.8380
Epoch 56/100
63/63 [==============================] - 29s 466ms/step - loss: 0.2311 - accuracy: 0.9010 - val_loss: 0.5791 - val_accuracy: 0.8210
Epoch 57/100
63/63 [==============================] - 29s 467ms/step - loss: 0.2442 - accuracy: 0.9020 - val_loss: 0.5084 - val_accuracy: 0.8410
Epoch 58/100
63/63 [==============================] - 29s 467ms/step - loss: 0.2555 - accuracy: 0.9005 - val_loss: 0.5380 - val_accuracy: 0.8460
Epoch 59/100
63/63 [==============================] - 30s 467ms/step - loss: 0.2510 - accuracy: 0.9060 - val_loss: 0.5833 - val_accuracy: 0.8030
Epoch 60/100
63/63 [==============================] - 30s 469ms/step - loss: 0.2388 - accuracy: 0.9055 - val_loss: 0.4984 - val_accuracy: 0.8220
Epoch 61/100
63/63 [==============================] - 30s 481ms/step - loss: 0.2367 - accuracy: 0.9065 - val_loss: 0.5625 - val_accuracy: 0.8160
Epoch 62/100
63/63 [==============================] - 30s 470ms/step - loss: 0.2633 - accuracy: 0.9045 - val_loss: 0.5785 - val_accuracy: 0.8390
Epoch 63/100
63/63 [==============================] - 30s 472ms/step - loss: 0.2197 - accuracy: 0.9155 - val_loss: 0.5369 - val_accuracy: 0.8400
Epoch 64/100
63/63 [==============================] - 29s 465ms/step - loss: 0.2365 - accuracy: 0.9065 - val_loss: 0.5924 - val_accuracy: 0.8410
Epoch 65/100
63/63 [==============================] - 30s 469ms/step - loss: 0.2426 - accuracy: 0.8995 - val_loss: 0.4842 - val_accuracy: 0.8590
Epoch 66/100
63/63 [==============================] - 30s 469ms/step - loss: 0.2302 - accuracy: 0.9150 - val_loss: 0.6195 - val_accuracy: 0.8250
Epoch 67/100
63/63 [==============================] - 30s 479ms/step - loss: 0.2527 - accuracy: 0.9075 - val_loss: 0.5000 - val_accuracy: 0.8270
Epoch 68/100
63/63 [==============================] - 30s 482ms/step - loss: 0.2115 - accuracy: 0.9210 - val_loss: 0.4948 - val_accuracy: 0.8440
Epoch 69/100
63/63 [==============================] - 29s 465ms/step - loss: 0.1963 - accuracy: 0.9250 - val_loss: 0.4802 - val_accuracy: 0.8570
Epoch 70/100
63/63 [==============================] - 29s 466ms/step - loss: 0.1929 - accuracy: 0.9215 - val_loss: 0.7604 - val_accuracy: 0.8300
Epoch 71/100
63/63 [==============================] - 29s 461ms/step - loss: 0.2172 - accuracy: 0.9155 - val_loss: 0.7469 - val_accuracy: 0.7890
Epoch 72/100
63/63 [==============================] - 30s 469ms/step - loss: 0.2028 - accuracy: 0.9270 - val_loss: 0.6504 - val_accuracy: 0.8170
Epoch 73/100
63/63 [==============================] - 29s 467ms/step - loss: 0.2256 - accuracy: 0.9130 - val_loss: 0.6156 - val_accuracy: 0.8590
Epoch 74/100
63/63 [==============================] - 30s 469ms/step - loss: 0.2081 - accuracy: 0.9185 - val_loss: 0.5738 - val_accuracy: 0.8340
Epoch 75/100
63/63 [==============================] - 31s 485ms/step - loss: 0.2133 - accuracy: 0.9190 - val_loss: 0.8909 - val_accuracy: 0.7860
Epoch 76/100
63/63 [==============================] - 30s 469ms/step - loss: 0.2005 - accuracy: 0.9305 - val_loss: 0.7914 - val_accuracy: 0.8200
Epoch 77/100
63/63 [==============================] - 30s 482ms/step - loss: 0.2163 - accuracy: 0.9205 - val_loss: 0.7526 - val_accuracy: 0.8120
Epoch 78/100
63/63 [==============================] - 30s 471ms/step - loss: 0.1968 - accuracy: 0.9300 - val_loss: 0.5349 - val_accuracy: 0.8360
Epoch 79/100
63/63 [==============================] - 30s 479ms/step - loss: 0.2600 - accuracy: 0.9225 - val_loss: 0.6733 - val_accuracy: 0.8130
Epoch 80/100
63/63 [==============================] - 31s 485ms/step - loss: 0.2115 - accuracy: 0.9175 - val_loss: 0.5988 - val_accuracy: 0.8420
Epoch 81/100
63/63 [==============================] - 30s 474ms/step - loss: 0.1909 - accuracy: 0.9280 - val_loss: 0.6096 - val_accuracy: 0.8360
Epoch 82/100
63/63 [==============================] - 29s 462ms/step - loss: 0.1997 - accuracy: 0.9205 - val_loss: 0.7397 - val_accuracy: 0.8290
Epoch 83/100
63/63 [==============================] - 30s 478ms/step - loss: 0.1898 - accuracy: 0.9295 - val_loss: 0.7289 - val_accuracy: 0.8470
Epoch 84/100
63/63 [==============================] - 30s 480ms/step - loss: 0.2013 - accuracy: 0.9265 - val_loss: 0.7059 - val_accuracy: 0.8170
Epoch 85/100
63/63 [==============================] - 31s 491ms/step - loss: 0.2028 - accuracy: 0.9240 - val_loss: 0.4695 - val_accuracy: 0.8500
Epoch 86/100
63/63 [==============================] - 30s 472ms/step - loss: 0.1798 - accuracy: 0.9380 - val_loss: 0.6632 - val_accuracy: 0.8450
Epoch 87/100
63/63 [==============================] - 30s 472ms/step - loss: 0.1986 - accuracy: 0.9235 - val_loss: 0.7441 - val_accuracy: 0.8520
Epoch 88/100
63/63 [==============================] - 30s 479ms/step - loss: 0.1901 - accuracy: 0.9295 - val_loss: 0.8115 - val_accuracy: 0.8150
Epoch 89/100
63/63 [==============================] - 30s 474ms/step - loss: 0.1612 - accuracy: 0.9365 - val_loss: 1.9230 - val_accuracy: 0.7780
Epoch 90/100
63/63 [==============================] - 29s 464ms/step - loss: 0.2082 - accuracy: 0.9275 - val_loss: 1.1947 - val_accuracy: 0.7740
Epoch 91/100
63/63 [==============================] - 30s 474ms/step - loss: 0.1820 - accuracy: 0.9390 - val_loss: 0.5873 - val_accuracy: 0.8520
Epoch 92/100
63/63 [==============================] - 32s 501ms/step - loss: 0.2000 - accuracy: 0.9280 - val_loss: 0.8253 - val_accuracy: 0.8300
Epoch 93/100
63/63 [==============================] - 30s 482ms/step - loss: 0.1598 - accuracy: 0.9415 - val_loss: 0.7330 - val_accuracy: 0.8540
Epoch 94/100
63/63 [==============================] - 30s 473ms/step - loss: 0.2021 - accuracy: 0.9315 - val_loss: 0.6678 - val_accuracy: 0.8450
Epoch 95/100
63/63 [==============================] - 32s 502ms/step - loss: 0.2164 - accuracy: 0.9290 - val_loss: 0.5134 - val_accuracy: 0.8570
Epoch 96/100
63/63 [==============================] - 31s 486ms/step - loss: 0.1825 - accuracy: 0.9360 - val_loss: 1.0677 - val_accuracy: 0.8280
Epoch 97/100
63/63 [==============================] - 30s 473ms/step - loss: 0.2125 - accuracy: 0.9360 - val_loss: 0.6128 - val_accuracy: 0.8640
Epoch 98/100
63/63 [==============================] - 30s 468ms/step - loss: 0.1789 - accuracy: 0.9315 - val_loss: 0.7538 - val_accuracy: 0.8420
Epoch 99/100
63/63 [==============================] - 30s 475ms/step - loss: 0.1762 - accuracy: 0.9340 - val_loss: 0.6393 - val_accuracy: 0.8320
Epoch 100/100
63/63 [==============================] - 29s 467ms/step - loss: 0.1499 - accuracy: 0.9455 - val_loss: 0.6866 - val_accuracy: 0.8490

import os
print(os.path.exists("./models/convnet_from_scratch_with_augmentation.h5"))

True

model.save("./models/test_manual_save.h5")
test_model = keras.models.load_model("./models/test_manual_save.h5")
print("Manual save and load successful!")

Manual save and load successful!

accuracy = history.history["accuracy"]
val_accuracy = history.history["val_accuracy"]
loss = history.history["loss"]
val_loss = history.history["val_loss"]
epochs = range(1, len(accuracy) + 1)
plt.plot(epochs, accuracy, "bo", label="Training accuracy")
plt.plot(epochs, val_accuracy, "b", label="Validation accuracy")
plt.title("Training and validation accuracy")
plt.legend()
plt.figure()
plt.plot(epochs, loss, "bo", label="Training loss")
plt.plot(epochs, val_loss, "b", label="Validation loss")
plt.title("Training and validation loss")
plt.legend()
plt.show()

Applying data augmentation significantly improved the custom CNN model’s performance by introducing variability through techniques like random flipping, rotations, and zoom. This enhanced the model’s generalization ability, reducing overfitting and increasing validation accuracy. The final validation accuracy surpassed the baseline, and validation loss stabilized, reflecting better adaptation to diverse scenarios. Augmentation proved essential for handling variability in the dataset, making the model more robust and effective in real-world applications. This demonstrates the importance of augmentation in achieving improved and consistent results in image classification tasks.

VGG

conv_base = keras.applications.vgg16.VGG16(
    weights="imagenet",
    include_top=False,
    input_shape=(180, 180, 3)
)

conv_base.summary()

Model: "vgg16"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 input_3 (InputLayer)        [(None, 180, 180, 3)]     0         
                                                                 
 block1_conv1 (Conv2D)       (None, 180, 180, 64)      1792      
                                                                 
 block1_conv2 (Conv2D)       (None, 180, 180, 64)      36928     
                                                                 
 block1_pool (MaxPooling2D)  (None, 90, 90, 64)        0         
                                                                 
 block2_conv1 (Conv2D)       (None, 90, 90, 128)       73856     
                                                                 
 block2_conv2 (Conv2D)       (None, 90, 90, 128)       147584    
                                                                 
 block2_pool (MaxPooling2D)  (None, 45, 45, 128)       0         
                                                                 
 block3_conv1 (Conv2D)       (None, 45, 45, 256)       295168    
                                                                 
 block3_conv2 (Conv2D)       (None, 45, 45, 256)       590080    
                                                                 
 block3_conv3 (Conv2D)       (None, 45, 45, 256)       590080    
                                                                 
 block3_pool (MaxPooling2D)  (None, 22, 22, 256)       0         
                                                                 
 block4_conv1 (Conv2D)       (None, 22, 22, 512)       1180160   
                                                                 
 block4_conv2 (Conv2D)       (None, 22, 22, 512)       2359808   
                                                                 
 block4_conv3 (Conv2D)       (None, 22, 22, 512)       2359808   
                                                                 
 block4_pool (MaxPooling2D)  (None, 11, 11, 512)       0         
                                                                 
 block5_conv1 (Conv2D)       (None, 11, 11, 512)       2359808   
                                                                 
 block5_conv2 (Conv2D)       (None, 11, 11, 512)       2359808   
                                                                 
 block5_conv3 (Conv2D)       (None, 11, 11, 512)       2359808   
                                                                 
 block5_pool (MaxPooling2D)  (None, 5, 5, 512)         0         
                                                                 
=================================================================
Total params: 14714688 (56.13 MB)
Trainable params: 14714688 (56.13 MB)
Non-trainable params: 0 (0.00 Byte)
_________________________________________________________________

import numpy as np

def get_features_and_labels(dataset):
    all_features = []
    all_labels = []
    for images, labels in dataset:
        preprocessed_images = keras.applications.vgg16.preprocess_input(images)
        features = conv_base.predict(preprocessed_images)
        all_features.append(features)
        all_labels.append(labels)
    return np.concatenate(all_features), np.concatenate(all_labels)

train_features, train_labels =  get_features_and_labels(train_dataset)
val_features, val_labels =  get_features_and_labels(validation_dataset)
test_features, test_labels =  get_features_and_labels(test_dataset)

1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 3s 3s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 1s 1s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 0s 488ms/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 2s 2s/step
1/1 [==============================] - 1s 995ms/step

train_features.shape

(2000, 5, 5, 512)

inputs = keras.Input(shape=(5, 5, 512))
x = layers.Flatten()(inputs)
x = layers.Dense(256)(x)
x = layers.Dropout(0.5)(x)
outputs = layers.Dense(1, activation="sigmoid")(x)
model = keras.Model(inputs, outputs)

model.summary()

Model: "model_2"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 input_4 (InputLayer)        [(None, 5, 5, 512)]       0         
                                                                 
 flatten_2 (Flatten)         (None, 12800)             0         
                                                                 
 dense_2 (Dense)             (None, 256)               3277056   
                                                                 
 dropout_1 (Dropout)         (None, 256)               0         
                                                                 
 dense_3 (Dense)             (None, 1)                 257       
                                                                 
=================================================================
Total params: 3277313 (12.50 MB)
Trainable params: 3277313 (12.50 MB)
Non-trainable params: 0 (0.00 Byte)
_________________________________________________________________

model_dir = "./models"
if not os.path.exists(model_dir):
    os.makedirs(model_dir)

# Compile the model
model.compile(
    loss="binary_crossentropy",
    optimizer="rmsprop",
    metrics=["accuracy"]
)

# Callback to save all models during training
callbacks = [
    ModelCheckpoint(
        filepath=f"{model_dir}/feature_extraction_epoch_{{epoch:02d}}.keras",  # Save all models
        save_best_only=False,                                                # Save every epoch
        verbose=1                                                            # Print save status
    )
]

# Train the model and save all versions
history = model.fit(
    train_features, train_labels,
    epochs=20,
    validation_data=(val_features, val_labels),
    callbacks=callbacks
)

# Extract validation losses
val_losses = history.history['val_loss']

# Identify the best epoch (lowest validation loss)
best_epoch = np.argmin(val_losses) + 1  # Add 1 because epochs are 1-indexed
print(f"Best epoch: {best_epoch} with validation loss: {val_losses[best_epoch - 1]}")

# Load the best model
best_model_path_vgg = f"{model_dir}/feature_extraction_epoch_{best_epoch:02d}.keras"
best_model_vgg = load_model(best_model_path_vgg, compile=False)
print(f"Loaded best model from {best_model_path}")

# Save the best model for deployment
best_model.save(f"{model_dir}/best_model_overall.h5")
print(f"Best model saved as '{model_dir}/best_model_overall.h5'")

Epoch 1/20
60/63 [===========================>..] - ETA: 0s - loss: 14.6337 - accuracy: 0.9219
Epoch 1: saving model to ./models/feature_extraction_epoch_01.keras
63/63 [==============================] - 1s 12ms/step - loss: 14.1999 - accuracy: 0.9240 - val_loss: 3.9698 - val_accuracy: 0.9640
Epoch 2/20
61/63 [============================>.] - ETA: 0s - loss: 4.3079 - accuracy: 0.9703
Epoch 2: saving model to ./models/feature_extraction_epoch_02.keras
63/63 [==============================] - 1s 10ms/step - loss: 4.2045 - accuracy: 0.9710 - val_loss: 4.4503 - val_accuracy: 0.9770
Epoch 3/20
61/63 [============================>.] - ETA: 0s - loss: 1.7259 - accuracy: 0.9867
Epoch 3: saving model to ./models/feature_extraction_epoch_03.keras
63/63 [==============================] - 1s 10ms/step - loss: 1.7749 - accuracy: 0.9865 - val_loss: 3.7269 - val_accuracy: 0.9780
Epoch 4/20
61/63 [============================>.] - ETA: 0s - loss: 1.6069 - accuracy: 0.9892
Epoch 4: saving model to ./models/feature_extraction_epoch_04.keras
63/63 [==============================] - 1s 10ms/step - loss: 1.5856 - accuracy: 0.9890 - val_loss: 3.9584 - val_accuracy: 0.9690
Epoch 5/20
61/63 [============================>.] - ETA: 0s - loss: 0.8307 - accuracy: 0.9908
Epoch 5: saving model to ./models/feature_extraction_epoch_05.keras
63/63 [==============================] - 1s 10ms/step - loss: 0.8108 - accuracy: 0.9910 - val_loss: 3.1451 - val_accuracy: 0.9780
Epoch 6/20
60/63 [===========================>..] - ETA: 0s - loss: 0.3942 - accuracy: 0.9979
Epoch 6: saving model to ./models/feature_extraction_epoch_06.keras
63/63 [==============================] - 1s 11ms/step - loss: 0.3784 - accuracy: 0.9980 - val_loss: 3.1805 - val_accuracy: 0.9840
Epoch 7/20
62/63 [============================>.] - ETA: 0s - loss: 0.5311 - accuracy: 0.9945
Epoch 7: saving model to ./models/feature_extraction_epoch_07.keras
63/63 [==============================] - 1s 11ms/step - loss: 0.5268 - accuracy: 0.9945 - val_loss: 6.4332 - val_accuracy: 0.9650
Epoch 8/20
61/63 [============================>.] - ETA: 0s - loss: 1.0330 - accuracy: 0.9923
Epoch 8: saving model to ./models/feature_extraction_epoch_08.keras
63/63 [==============================] - 1s 11ms/step - loss: 1.1229 - accuracy: 0.9920 - val_loss: 3.9692 - val_accuracy: 0.9790
Epoch 9/20
58/63 [==========================>...] - ETA: 0s - loss: 0.2640 - accuracy: 0.9957
Epoch 9: saving model to ./models/feature_extraction_epoch_09.keras
63/63 [==============================] - 1s 11ms/step - loss: 0.2450 - accuracy: 0.9960 - val_loss: 4.0796 - val_accuracy: 0.9810
Epoch 10/20
60/63 [===========================>..] - ETA: 0s - loss: 0.2005 - accuracy: 0.9990
Epoch 10: saving model to ./models/feature_extraction_epoch_10.keras
63/63 [==============================] - 1s 11ms/step - loss: 0.2829 - accuracy: 0.9985 - val_loss: 7.7341 - val_accuracy: 0.9580
Epoch 11/20
60/63 [===========================>..] - ETA: 0s - loss: 0.5286 - accuracy: 0.9958
Epoch 11: saving model to ./models/feature_extraction_epoch_11.keras
63/63 [==============================] - 1s 10ms/step - loss: 0.5075 - accuracy: 0.9960 - val_loss: 4.8037 - val_accuracy: 0.9790
Epoch 12/20
61/63 [============================>.] - ETA: 0s - loss: 0.0063 - accuracy: 0.9995
Epoch 12: saving model to ./models/feature_extraction_epoch_12.keras
63/63 [==============================] - 1s 10ms/step - loss: 0.0061 - accuracy: 0.9995 - val_loss: 3.8714 - val_accuracy: 0.9830
Epoch 13/20
61/63 [============================>.] - ETA: 0s - loss: 0.0710 - accuracy: 0.9990  
Epoch 13: saving model to ./models/feature_extraction_epoch_13.keras
63/63 [==============================] - 1s 10ms/step - loss: 0.0693 - accuracy: 0.9990 - val_loss: 4.3214 - val_accuracy: 0.9780
Epoch 14/20
62/63 [============================>.] - ETA: 0s - loss: 0.1744 - accuracy: 0.9990
Epoch 14: saving model to ./models/feature_extraction_epoch_14.keras
63/63 [==============================] - 1s 11ms/step - loss: 0.1730 - accuracy: 0.9990 - val_loss: 4.2282 - val_accuracy: 0.9790
Epoch 15/20
61/63 [============================>.] - ETA: 0s - loss: 8.4908e-10 - accuracy: 1.0000
Epoch 15: saving model to ./models/feature_extraction_epoch_15.keras
63/63 [==============================] - 1s 10ms/step - loss: 8.2870e-10 - accuracy: 1.0000 - val_loss: 4.0219 - val_accuracy: 0.9780
Epoch 16/20
60/63 [===========================>..] - ETA: 0s - loss: 0.2684 - accuracy: 0.9964
Epoch 16: saving model to ./models/feature_extraction_epoch_16.keras
63/63 [==============================] - 1s 10ms/step - loss: 0.2577 - accuracy: 0.9965 - val_loss: 4.7184 - val_accuracy: 0.9760
Epoch 17/20
61/63 [============================>.] - ETA: 0s - loss: 0.2278 - accuracy: 0.9985
Epoch 17: saving model to ./models/feature_extraction_epoch_17.keras
63/63 [==============================] - 1s 10ms/step - loss: 0.2223 - accuracy: 0.9985 - val_loss: 5.2137 - val_accuracy: 0.9750
Epoch 18/20
61/63 [============================>.] - ETA: 0s - loss: 0.0704 - accuracy: 0.9990
Epoch 18: saving model to ./models/feature_extraction_epoch_18.keras
63/63 [==============================] - 1s 10ms/step - loss: 0.0687 - accuracy: 0.9990 - val_loss: 5.0982 - val_accuracy: 0.9760
Epoch 19/20
63/63 [==============================] - ETA: 0s - loss: 1.2685e-09 - accuracy: 1.0000
Epoch 19: saving model to ./models/feature_extraction_epoch_19.keras
63/63 [==============================] - 1s 11ms/step - loss: 1.2685e-09 - accuracy: 1.0000 - val_loss: 4.9652 - val_accuracy: 0.9770
Epoch 20/20
61/63 [============================>.] - ETA: 0s - loss: 0.0766 - accuracy: 0.9990
Epoch 20: saving model to ./models/feature_extraction_epoch_20.keras
63/63 [==============================] - 1s 10ms/step - loss: 0.0747 - accuracy: 0.9990 - val_loss: 4.1229 - val_accuracy: 0.9780
Best epoch: 5 with validation loss: 3.1451103687286377
Loaded best model from ./models/model_epoch_10.h5
Best model saved as './models/best_model_overall.h5'

import matplotlib.pyplot as plt
acc = history.history["accuracy"]
val_acc = history.history["val_accuracy"]
loss = history.history["loss"]
val_loss = history.history["val_loss"]
epochs = range(1, len(acc) + 1)
plt.plot(epochs, acc, "bo", label="Training accuracy")
plt.plot(epochs, val_acc, "b", label="Validation accuracy")
plt.title("Training and validation accuracy")
plt.legend()
plt.figure()
plt.plot(epochs, loss, "bo", label="Training loss")
plt.plot(epochs, val_loss, "b", label="Validation loss")
plt.title("Training and validation loss")
plt.legend()
plt.show()

test_model = keras.models.load_model(
    "./models/feature_extraction.keras.h5")
test_loss, test_acc = test_model.evaluate(x=test_features, y=test_labels)
print(f"Test accuracy: {test_acc:.3f}")

63/63 [==============================] - 0s 2ms/step - loss: 4.4031 - accuracy: 0.9720
Test accuracy: 0.972

The first experiment with the VGG backbone, excluding the top layers, demonstrated impressive results, achieving a final test accuracy of 97.2% with a validation accuracy peak at 97.8% (Epoch 5). As seen in the accuracy graph, the model quickly stabilized, while the loss graph shows a steep initial drop in training loss, followed by minor fluctuations in validation loss. This indicates the model’s ability to leverage pre-trained feature extraction effectively. Although overfitting was minimal, early stopping was vital to capture the best performance. These findings emphasize the efficiency of using pre-trained architectures for feature extraction.

Feature extraction together with data augmentation

conv_base  = keras.applications.vgg16.VGG16(
    weights="imagenet",
    include_top=False)

conv_base.summary()

Model: "vgg16"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 input_5 (InputLayer)        [(None, None, None, 3)]   0         
                                                                 
 block1_conv1 (Conv2D)       (None, None, None, 64)    1792      
                                                                 
 block1_conv2 (Conv2D)       (None, None, None, 64)    36928     
                                                                 
 block1_pool (MaxPooling2D)  (None, None, None, 64)    0         
                                                                 
 block2_conv1 (Conv2D)       (None, None, None, 128)   73856     
                                                                 
 block2_conv2 (Conv2D)       (None, None, None, 128)   147584    
                                                                 
 block2_pool (MaxPooling2D)  (None, None, None, 128)   0         
                                                                 
 block3_conv1 (Conv2D)       (None, None, None, 256)   295168    
                                                                 
 block3_conv2 (Conv2D)       (None, None, None, 256)   590080    
                                                                 
 block3_conv3 (Conv2D)       (None, None, None, 256)   590080    
                                                                 
 block3_pool (MaxPooling2D)  (None, None, None, 256)   0         
                                                                 
 block4_conv1 (Conv2D)       (None, None, None, 512)   1180160   
                                                                 
 block4_conv2 (Conv2D)       (None, None, None, 512)   2359808   
                                                                 
 block4_conv3 (Conv2D)       (None, None, None, 512)   2359808   
                                                                 
 block4_pool (MaxPooling2D)  (None, None, None, 512)   0         
                                                                 
 block5_conv1 (Conv2D)       (None, None, None, 512)   2359808   
                                                                 
 block5_conv2 (Conv2D)       (None, None, None, 512)   2359808   
                                                                 
 block5_conv3 (Conv2D)       (None, None, None, 512)   2359808   
                                                                 
 block5_pool (MaxPooling2D)  (None, None, None, 512)   0         
                                                                 
=================================================================
Total params: 14714688 (56.13 MB)
Trainable params: 14714688 (56.13 MB)
Non-trainable params: 0 (0.00 Byte)
_________________________________________________________________

conv_base.trainable = False
conv_base.summary()

Model: "vgg16"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 input_5 (InputLayer)        [(None, None, None, 3)]   0         
                                                                 
 block1_conv1 (Conv2D)       (None, None, None, 64)    1792      
                                                                 
 block1_conv2 (Conv2D)       (None, None, None, 64)    36928     
                                                                 
 block1_pool (MaxPooling2D)  (None, None, None, 64)    0         
                                                                 
 block2_conv1 (Conv2D)       (None, None, None, 128)   73856     
                                                                 
 block2_conv2 (Conv2D)       (None, None, None, 128)   147584    
                                                                 
 block2_pool (MaxPooling2D)  (None, None, None, 128)   0         
                                                                 
 block3_conv1 (Conv2D)       (None, None, None, 256)   295168    
                                                                 
 block3_conv2 (Conv2D)       (None, None, None, 256)   590080    
                                                                 
 block3_conv3 (Conv2D)       (None, None, None, 256)   590080    
                                                                 
 block3_pool (MaxPooling2D)  (None, None, None, 256)   0         
                                                                 
 block4_conv1 (Conv2D)       (None, None, None, 512)   1180160   
                                                                 
 block4_conv2 (Conv2D)       (None, None, None, 512)   2359808   
                                                                 
 block4_conv3 (Conv2D)       (None, None, None, 512)   2359808   
                                                                 
 block4_pool (MaxPooling2D)  (None, None, None, 512)   0         
                                                                 
 block5_conv1 (Conv2D)       (None, None, None, 512)   2359808   
                                                                 
 block5_conv2 (Conv2D)       (None, None, None, 512)   2359808   
                                                                 
 block5_conv3 (Conv2D)       (None, None, None, 512)   2359808   
                                                                 
 block5_pool (MaxPooling2D)  (None, None, None, 512)   0         
                                                                 
=================================================================
Total params: 14714688 (56.13 MB)
Trainable params: 0 (0.00 Byte)
Non-trainable params: 14714688 (56.13 MB)
_________________________________________________________________

data_augmentation = keras.Sequential(
    [
        layers.RandomFlip("horizontal"),
        layers.RandomRotation(0.1),
        layers.RandomZoom(0.2),
    ]
)

inputs = keras.Input(shape=(180, 180, 3))
x = data_augmentation(inputs)
x = keras.applications.vgg16.preprocess_input(x)
x = conv_base(x)
x = layers.Flatten()(x)
x = layers.Dense(256)(x)
x = layers.Dropout(0.5)(x)
outputs = layers.Dense(1, activation="sigmoid")(x)
model = keras.Model(inputs, outputs)

model.summary()

Model: "model_3"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 input_6 (InputLayer)        [(None, 180, 180, 3)]     0         
                                                                 
 sequential_1 (Sequential)   (None, 180, 180, 3)       0         
                                                                 
 tf.__operators__.getitem (  (None, 180, 180, 3)       0         
 SlicingOpLambda)                                                
                                                                 
 tf.nn.bias_add (TFOpLambda  (None, 180, 180, 3)       0         
 )                                                               
                                                                 
 vgg16 (Functional)          (None, None, None, 512)   14714688  
                                                                 
 flatten_3 (Flatten)         (None, 12800)             0         
                                                                 
 dense_4 (Dense)             (None, 256)               3277056   
                                                                 
 dropout_2 (Dropout)         (None, 256)               0         
                                                                 
 dense_5 (Dense)             (None, 1)                 257       
                                                                 
=================================================================
Total params: 17992001 (68.63 MB)
Trainable params: 3277313 (12.50 MB)
Non-trainable params: 14714688 (56.13 MB)
_________________________________________________________________

model.compile(loss="binary_crossentropy",
              optimizer="rmsprop",
              metrics=["accuracy"])

callbacks = [
    keras.callbacks.ModelCheckpoint(
        filepath="./models/feature_extraction_with_data_augmentation.keras.h5",
        save_best_only=True,
        monitor="val_loss")
]
history = model.fit(
    train_dataset,
    epochs=50,
    validation_data=validation_dataset,
    callbacks=callbacks)

Epoch 1/50
63/63 [==============================] - 194s 3s/step - loss: 17.2915 - accuracy: 0.8945 - val_loss: 4.9827 - val_accuracy: 0.9720
Epoch 2/50
63/63 [==============================] - 194s 3s/step - loss: 8.0896 - accuracy: 0.9400 - val_loss: 3.7733 - val_accuracy: 0.9700
Epoch 3/50
63/63 [==============================] - 194s 3s/step - loss: 7.3365 - accuracy: 0.9465 - val_loss: 5.4666 - val_accuracy: 0.9690
Epoch 4/50
63/63 [==============================] - 197s 3s/step - loss: 3.9339 - accuracy: 0.9630 - val_loss: 3.4692 - val_accuracy: 0.9780
Epoch 5/50
63/63 [==============================] - 198s 3s/step - loss: 3.9868 - accuracy: 0.9685 - val_loss: 3.5862 - val_accuracy: 0.9790
Epoch 6/50
63/63 [==============================] - 197s 3s/step - loss: 4.1430 - accuracy: 0.9670 - val_loss: 4.8031 - val_accuracy: 0.9680
Epoch 7/50
63/63 [==============================] - 198s 3s/step - loss: 5.4086 - accuracy: 0.9615 - val_loss: 4.3880 - val_accuracy: 0.9770
Epoch 8/50
63/63 [==============================] - 199s 3s/step - loss: 3.2938 - accuracy: 0.9740 - val_loss: 3.1562 - val_accuracy: 0.9810
Epoch 9/50
63/63 [==============================] - 198s 3s/step - loss: 2.8142 - accuracy: 0.9730 - val_loss: 3.0806 - val_accuracy: 0.9770
Epoch 10/50
63/63 [==============================] - 201s 3s/step - loss: 1.9774 - accuracy: 0.9775 - val_loss: 6.8125 - val_accuracy: 0.9660
Epoch 11/50
63/63 [==============================] - 199s 3s/step - loss: 2.3032 - accuracy: 0.9785 - val_loss: 4.3972 - val_accuracy: 0.9750
Epoch 12/50
63/63 [==============================] - 201s 3s/step - loss: 2.4813 - accuracy: 0.9780 - val_loss: 3.4271 - val_accuracy: 0.9800
Epoch 13/50
63/63 [==============================] - 198s 3s/step - loss: 2.2144 - accuracy: 0.9790 - val_loss: 3.0805 - val_accuracy: 0.9750
Epoch 14/50
63/63 [==============================] - 200s 3s/step - loss: 2.5829 - accuracy: 0.9750 - val_loss: 3.5703 - val_accuracy: 0.9760
Epoch 15/50
63/63 [==============================] - 198s 3s/step - loss: 1.9714 - accuracy: 0.9780 - val_loss: 4.0435 - val_accuracy: 0.9780
Epoch 16/50
63/63 [==============================] - 197s 3s/step - loss: 2.1867 - accuracy: 0.9790 - val_loss: 3.3067 - val_accuracy: 0.9800
Epoch 17/50
63/63 [==============================] - 200s 3s/step - loss: 1.7393 - accuracy: 0.9825 - val_loss: 3.1684 - val_accuracy: 0.9790
Epoch 18/50
63/63 [==============================] - 203s 3s/step - loss: 1.7996 - accuracy: 0.9780 - val_loss: 3.2219 - val_accuracy: 0.9800
Epoch 19/50
63/63 [==============================] - 198s 3s/step - loss: 1.5938 - accuracy: 0.9820 - val_loss: 4.0213 - val_accuracy: 0.9730
Epoch 20/50
63/63 [==============================] - 202s 3s/step - loss: 1.1281 - accuracy: 0.9870 - val_loss: 3.1636 - val_accuracy: 0.9750
Epoch 21/50
63/63 [==============================] - 204s 3s/step - loss: 1.7154 - accuracy: 0.9825 - val_loss: 3.6919 - val_accuracy: 0.9770
Epoch 22/50
63/63 [==============================] - 201s 3s/step - loss: 0.8278 - accuracy: 0.9865 - val_loss: 3.6382 - val_accuracy: 0.9750
Epoch 23/50
63/63 [==============================] - 200s 3s/step - loss: 1.3542 - accuracy: 0.9820 - val_loss: 2.0926 - val_accuracy: 0.9780
Epoch 24/50
63/63 [==============================] - 200s 3s/step - loss: 0.7972 - accuracy: 0.9875 - val_loss: 1.6941 - val_accuracy: 0.9790
Epoch 25/50
63/63 [==============================] - 204s 3s/step - loss: 0.8696 - accuracy: 0.9880 - val_loss: 2.1241 - val_accuracy: 0.9810
Epoch 26/50
63/63 [==============================] - 201s 3s/step - loss: 1.1562 - accuracy: 0.9840 - val_loss: 2.0646 - val_accuracy: 0.9830
Epoch 27/50
63/63 [==============================] - 200s 3s/step - loss: 1.1376 - accuracy: 0.9865 - val_loss: 1.5391 - val_accuracy: 0.9860
Epoch 28/50
63/63 [==============================] - 198s 3s/step - loss: 0.8379 - accuracy: 0.9880 - val_loss: 2.0473 - val_accuracy: 0.9830
Epoch 29/50
63/63 [==============================] - 200s 3s/step - loss: 1.0329 - accuracy: 0.9905 - val_loss: 1.8395 - val_accuracy: 0.9860
Epoch 30/50
63/63 [==============================] - 197s 3s/step - loss: 0.7264 - accuracy: 0.9875 - val_loss: 2.7718 - val_accuracy: 0.9840
Epoch 31/50
63/63 [==============================] - 199s 3s/step - loss: 0.9439 - accuracy: 0.9870 - val_loss: 2.2218 - val_accuracy: 0.9820
Epoch 32/50
63/63 [==============================] - 201s 3s/step - loss: 0.6827 - accuracy: 0.9905 - val_loss: 1.8822 - val_accuracy: 0.9830
Epoch 33/50
63/63 [==============================] - 200s 3s/step - loss: 0.8924 - accuracy: 0.9885 - val_loss: 1.9310 - val_accuracy: 0.9810
Epoch 34/50
63/63 [==============================] - 208s 3s/step - loss: 0.6922 - accuracy: 0.9895 - val_loss: 1.9256 - val_accuracy: 0.9790
Epoch 35/50
63/63 [==============================] - 200s 3s/step - loss: 0.5699 - accuracy: 0.9890 - val_loss: 2.3065 - val_accuracy: 0.9750
Epoch 36/50
63/63 [==============================] - 207s 3s/step - loss: 0.8880 - accuracy: 0.9850 - val_loss: 2.4384 - val_accuracy: 0.9780
Epoch 37/50
63/63 [==============================] - 204s 3s/step - loss: 0.9261 - accuracy: 0.9840 - val_loss: 3.1480 - val_accuracy: 0.9730
Epoch 38/50
63/63 [==============================] - 224s 4s/step - loss: 0.6328 - accuracy: 0.9880 - val_loss: 3.1115 - val_accuracy: 0.9740
Epoch 39/50
63/63 [==============================] - 211s 3s/step - loss: 0.3189 - accuracy: 0.9910 - val_loss: 2.6314 - val_accuracy: 0.9800
Epoch 40/50
63/63 [==============================] - 195s 3s/step - loss: 0.7681 - accuracy: 0.9895 - val_loss: 2.8807 - val_accuracy: 0.9760
Epoch 41/50
63/63 [==============================] - 193s 3s/step - loss: 0.9178 - accuracy: 0.9875 - val_loss: 2.0761 - val_accuracy: 0.9790
Epoch 42/50
63/63 [==============================] - 195s 3s/step - loss: 0.4202 - accuracy: 0.9930 - val_loss: 2.2287 - val_accuracy: 0.9780
Epoch 43/50
63/63 [==============================] - 195s 3s/step - loss: 0.5040 - accuracy: 0.9890 - val_loss: 2.1872 - val_accuracy: 0.9810
Epoch 44/50
63/63 [==============================] - 197s 3s/step - loss: 0.6028 - accuracy: 0.9880 - val_loss: 2.2335 - val_accuracy: 0.9780
Epoch 45/50
63/63 [==============================] - 198s 3s/step - loss: 0.9805 - accuracy: 0.9875 - val_loss: 3.8656 - val_accuracy: 0.9740
Epoch 46/50
63/63 [==============================] - 196s 3s/step - loss: 0.5861 - accuracy: 0.9900 - val_loss: 2.9878 - val_accuracy: 0.9760
Epoch 47/50
63/63 [==============================] - 198s 3s/step - loss: 0.5361 - accuracy: 0.9920 - val_loss: 2.9417 - val_accuracy: 0.9760
Epoch 48/50
63/63 [==============================] - 200s 3s/step - loss: 0.5606 - accuracy: 0.9890 - val_loss: 2.2579 - val_accuracy: 0.9750
Epoch 49/50
63/63 [==============================] - 197s 3s/step - loss: 0.6216 - accuracy: 0.9890 - val_loss: 2.8632 - val_accuracy: 0.9780
Epoch 50/50
63/63 [==============================] - 198s 3s/step - loss: 0.8053 - accuracy: 0.9845 - val_loss: 2.8266 - val_accuracy: 0.9730

acc = history.history["accuracy"]
val_acc = history.history["val_accuracy"]
loss = history.history["loss"]
val_loss = history.history["val_loss"]
epochs = range(1, len(acc) + 1)
plt.plot(epochs, acc, "bo", label="Training accuracy")
plt.plot(epochs, val_acc, "b", label="Validation accuracy")
plt.title("Training and validation accuracy")
plt.legend()
plt.figure()
plt.plot(epochs, loss, "bo", label="Training loss")
plt.plot(epochs, val_loss, "b", label="Validation loss")
plt.title("Training and validation loss")
plt.legend()
plt.show()

test_model = keras.models.load_model(
    "./models/feature_extraction_with_data_augmentation.keras.h5")
test_loss, test_acc = test_model.evaluate(test_dataset)
print(f"Test accuracy: {test_acc:.3f}")

63/63 [==============================] - 130s 2s/step - loss: 3.3138 - accuracy: 0.9740
Test accuracy: 0.974

The feature extraction experiment with augmentation further improved the VGG model’s performance, achieving a test accuracy of 97.4%. As shown in the accuracy graph, the model consistently performed well across 50 epochs, maintaining high validation accuracy with minor fluctuations. The loss graph reveals a steep decline in training loss early on, stabilizing around epoch 10, while validation loss showed less variability compared to the initial VGG experiment.

Fine-tuning a pretrained model

conv_base.summary()

Model: "vgg16"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 input_5 (InputLayer)        [(None, None, None, 3)]   0         
                                                                 
 block1_conv1 (Conv2D)       (None, None, None, 64)    1792      
                                                                 
 block1_conv2 (Conv2D)       (None, None, None, 64)    36928     
                                                                 
 block1_pool (MaxPooling2D)  (None, None, None, 64)    0         
                                                                 
 block2_conv1 (Conv2D)       (None, None, None, 128)   73856     
                                                                 
 block2_conv2 (Conv2D)       (None, None, None, 128)   147584    
                                                                 
 block2_pool (MaxPooling2D)  (None, None, None, 128)   0         
                                                                 
 block3_conv1 (Conv2D)       (None, None, None, 256)   295168    
                                                                 
 block3_conv2 (Conv2D)       (None, None, None, 256)   590080    
                                                                 
 block3_conv3 (Conv2D)       (None, None, None, 256)   590080    
                                                                 
 block3_pool (MaxPooling2D)  (None, None, None, 256)   0         
                                                                 
 block4_conv1 (Conv2D)       (None, None, None, 512)   1180160   
                                                                 
 block4_conv2 (Conv2D)       (None, None, None, 512)   2359808   
                                                                 
 block4_conv3 (Conv2D)       (None, None, None, 512)   2359808   
                                                                 
 block4_pool (MaxPooling2D)  (None, None, None, 512)   0         
                                                                 
 block5_conv1 (Conv2D)       (None, None, None, 512)   2359808   
                                                                 
 block5_conv2 (Conv2D)       (None, None, None, 512)   2359808   
                                                                 
 block5_conv3 (Conv2D)       (None, None, None, 512)   2359808   
                                                                 
 block5_pool (MaxPooling2D)  (None, None, None, 512)   0         
                                                                 
=================================================================
Total params: 14714688 (56.13 MB)
Trainable params: 0 (0.00 Byte)
Non-trainable params: 14714688 (56.13 MB)
_________________________________________________________________

conv_base.trainable = True
for layer in conv_base.layers[:-4]:
    layer.trainable = False


model.summary()

Model: "model_3"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 input_6 (InputLayer)        [(None, 180, 180, 3)]     0         
                                                                 
 sequential_1 (Sequential)   (None, 180, 180, 3)       0         
                                                                 
 tf.__operators__.getitem (  (None, 180, 180, 3)       0         
 SlicingOpLambda)                                                
                                                                 
 tf.nn.bias_add (TFOpLambda  (None, 180, 180, 3)       0         
 )                                                               
                                                                 
 vgg16 (Functional)          (None, None, None, 512)   14714688  
                                                                 
 flatten_3 (Flatten)         (None, 12800)             0         
                                                                 
 dense_4 (Dense)             (None, 256)               3277056   
                                                                 
 dropout_2 (Dropout)         (None, 256)               0         
                                                                 
 dense_5 (Dense)             (None, 1)                 257       
                                                                 
=================================================================
Total params: 17992001 (68.63 MB)
Trainable params: 10356737 (39.51 MB)
Non-trainable params: 7635264 (29.13 MB)
_________________________________________________________________

model.compile(loss="binary_crossentropy",
              optimizer=keras.optimizers.RMSprop(learning_rate=1e-5),
              metrics=["accuracy"])

callbacks = [
    keras.callbacks.ModelCheckpoint(
        filepath="./models/fine_tuning.keras.h5",
        save_best_only=True,
        monitor="val_loss")
]
history = model.fit(
    train_dataset,
    epochs=30,
    validation_data=validation_dataset,
    callbacks=callbacks)

WARNING:absl:At this time, the v2.11+ optimizer `tf.keras.optimizers.RMSprop` runs slowly on M1/M2 Macs, please use the legacy Keras optimizer instead, located at `tf.keras.optimizers.legacy.RMSprop`.

Epoch 1/30
63/63 [==============================] - 226s 4s/step - loss: 0.8322 - accuracy: 0.9855 - val_loss: 2.5049 - val_accuracy: 0.9720
Epoch 2/30
63/63 [==============================] - 226s 4s/step - loss: 0.4245 - accuracy: 0.9895 - val_loss: 2.2815 - val_accuracy: 0.9740
Epoch 3/30
63/63 [==============================] - 229s 4s/step - loss: 0.3800 - accuracy: 0.9935 - val_loss: 1.8590 - val_accuracy: 0.9770
Epoch 4/30
63/63 [==============================] - 223s 4s/step - loss: 0.4245 - accuracy: 0.9855 - val_loss: 2.6160 - val_accuracy: 0.9710
Epoch 5/30
63/63 [==============================] - 226s 4s/step - loss: 0.2574 - accuracy: 0.9935 - val_loss: 1.7396 - val_accuracy: 0.9790
Epoch 6/30
63/63 [==============================] - 228s 4s/step - loss: 0.4529 - accuracy: 0.9890 - val_loss: 1.9925 - val_accuracy: 0.9800
Epoch 7/30
63/63 [==============================] - 234s 4s/step - loss: 0.3236 - accuracy: 0.9930 - val_loss: 1.8327 - val_accuracy: 0.9830
Epoch 8/30
63/63 [==============================] - 228s 4s/step - loss: 0.3008 - accuracy: 0.9925 - val_loss: 2.1597 - val_accuracy: 0.9810
Epoch 9/30
63/63 [==============================] - 221s 4s/step - loss: 0.2259 - accuracy: 0.9930 - val_loss: 2.0111 - val_accuracy: 0.9770
Epoch 10/30
63/63 [==============================] - 229s 4s/step - loss: 0.2475 - accuracy: 0.9925 - val_loss: 2.2725 - val_accuracy: 0.9750
Epoch 11/30
63/63 [==============================] - 229s 4s/step - loss: 0.1089 - accuracy: 0.9955 - val_loss: 2.2624 - val_accuracy: 0.9700
Epoch 12/30
63/63 [==============================] - 227s 4s/step - loss: 0.0860 - accuracy: 0.9950 - val_loss: 2.1484 - val_accuracy: 0.9780
Epoch 13/30
63/63 [==============================] - 226s 4s/step - loss: 0.2383 - accuracy: 0.9925 - val_loss: 2.1651 - val_accuracy: 0.9800
Epoch 14/30
63/63 [==============================] - 229s 4s/step - loss: 0.1358 - accuracy: 0.9940 - val_loss: 2.1366 - val_accuracy: 0.9730
Epoch 15/30
63/63 [==============================] - 232s 4s/step - loss: 0.0897 - accuracy: 0.9975 - val_loss: 1.9261 - val_accuracy: 0.9740
Epoch 16/30
63/63 [==============================] - 229s 4s/step - loss: 0.3557 - accuracy: 0.9925 - val_loss: 2.0569 - val_accuracy: 0.9760
Epoch 17/30
63/63 [==============================] - 231s 4s/step - loss: 0.0354 - accuracy: 0.9980 - val_loss: 1.9074 - val_accuracy: 0.9750
Epoch 18/30
63/63 [==============================] - 228s 4s/step - loss: 0.1532 - accuracy: 0.9965 - val_loss: 1.9789 - val_accuracy: 0.9780
Epoch 19/30
63/63 [==============================] - 232s 4s/step - loss: 0.1495 - accuracy: 0.9955 - val_loss: 2.5848 - val_accuracy: 0.9700
Epoch 20/30
63/63 [==============================] - 226s 4s/step - loss: 0.0860 - accuracy: 0.9960 - val_loss: 2.0887 - val_accuracy: 0.9780
Epoch 21/30
63/63 [==============================] - 236s 4s/step - loss: 0.1210 - accuracy: 0.9965 - val_loss: 2.2238 - val_accuracy: 0.9810
Epoch 22/30
63/63 [==============================] - 226s 4s/step - loss: 0.1791 - accuracy: 0.9940 - val_loss: 2.4960 - val_accuracy: 0.9820
Epoch 23/30
63/63 [==============================] - 226s 4s/step - loss: 0.1995 - accuracy: 0.9945 - val_loss: 2.0961 - val_accuracy: 0.9820
Epoch 24/30
63/63 [==============================] - 220s 4s/step - loss: 0.0812 - accuracy: 0.9985 - val_loss: 2.0610 - val_accuracy: 0.9830
Epoch 25/30
63/63 [==============================] - 219s 3s/step - loss: 0.2065 - accuracy: 0.9955 - val_loss: 1.6147 - val_accuracy: 0.9830
Epoch 26/30
63/63 [==============================] - 222s 4s/step - loss: 0.1346 - accuracy: 0.9950 - val_loss: 2.0668 - val_accuracy: 0.9830
Epoch 27/30
63/63 [==============================] - 222s 4s/step - loss: 0.0638 - accuracy: 0.9955 - val_loss: 1.9553 - val_accuracy: 0.9830
Epoch 28/30
63/63 [==============================] - 225s 4s/step - loss: 0.1611 - accuracy: 0.9955 - val_loss: 2.1722 - val_accuracy: 0.9770
Epoch 29/30
63/63 [==============================] - 217s 3s/step - loss: 0.0361 - accuracy: 0.9975 - val_loss: 2.0795 - val_accuracy: 0.9770
Epoch 30/30
63/63 [==============================] - 219s 3s/step - loss: 0.0485 - accuracy: 0.9980 - val_loss: 1.8098 - val_accuracy: 0.9800

acc = history.history["accuracy"]
val_acc = history.history["val_accuracy"]
loss = history.history["loss"]
val_loss = history.history["val_loss"]
epochs = range(1, len(acc) + 1)
plt.plot(epochs, acc, "bo", label="Training accuracy")
plt.plot(epochs, val_acc, "b", label="Validation accuracy")
plt.title("Training and validation accuracy")
plt.legend()
plt.figure()
plt.plot(epochs, loss, "bo", label="Training loss")
plt.plot(epochs, val_loss, "b", label="Validation loss")
plt.title("Training and validation loss")
plt.legend()
plt.show()

model = keras.models.load_model("./models/fine_tuning.keras.h5")
test_loss, test_acc = model.evaluate(test_dataset)
print(f"Test accuracy: {test_acc:.3f}")

WARNING:absl:At this time, the v2.11+ optimizer `tf.keras.optimizers.RMSprop` runs slowly on M1/M2 Macs, please use the legacy Keras optimizer instead, located at `tf.keras.optimizers.legacy.RMSprop`.

63/63 [==============================] - 140s 2s/step - loss: 2.2558 - accuracy: 0.9745
Test accuracy: 0.975

After fine-tuning the VGG model, the test accuracy further improved to 97.5%, demonstrating the effectiveness of this approach. As seen in the accuracy graph, both training and validation accuracies remained consistently high, with validation accuracy peaking at 98.3%. The loss graph shows stable training loss and slightly fluctuating validation loss, which stabilized towards the later epochs. Fine-tuning allowed the model to adapt pre-trained features more effectively to the task, leveraging additional trainable layers.

Final evaluations of best model of both networks

from tensorflow.keras.models import load_model
from tensorflow.keras.utils import image_dataset_from_directory
from sklearn.metrics import confusion_matrix, classification_report, precision_recall_curve, average_precision_score
import matplotlib.pyplot as plt
import numpy as np
import pathlib

# Define the path to your dataset
data_folder = pathlib.Path("../CSCN8010/data/kaggle_dogs_vs_cats_small")

# Initialize test dataset
test_dataset = image_dataset_from_directory(
    data_folder / "test",
    image_size=(180, 180),
    batch_size=32,
    shuffle=False
)

# Load the best models
vgg_model = load_model("./models/fine_tuning.keras.h5")
cnn_model = load_model("./models/convnet_from_scratch_with_augmentation.h5")

# Extract true labels and images from test_dataset
test_labels = np.concatenate([y.numpy() for x, y in test_dataset], axis=0)
test_images = np.concatenate([x.numpy() for x, y in test_dataset], axis=0)

# Get predictions and probabilities
vgg_preds = (vgg_model.predict(test_dataset) > 0.5).astype("int32").flatten()
cnn_preds = (cnn_model.predict(test_dataset) > 0.5).astype("int32").flatten()
vgg_probs = vgg_model.predict(test_dataset).flatten()
cnn_probs = cnn_model.predict(test_dataset).flatten()

Found 2000 files belonging to 2 classes.

WARNING:absl:At this time, the v2.11+ optimizer `tf.keras.optimizers.RMSprop` runs slowly on M1/M2 Macs, please use the legacy Keras optimizer instead, located at `tf.keras.optimizers.legacy.RMSprop`.

63/63 [==============================] - 136s 2s/step
63/63 [==============================] - 7s 116ms/step
63/63 [==============================] - 150s 2s/step
63/63 [==============================] - 7s 114ms/step

# Evaluate test accuracy
vgg_loss, vgg_acc = vgg_model.evaluate(test_dataset)
cnn_loss, cnn_acc = cnn_model.evaluate(test_dataset)

print(f"VGG Model Accuracy: {vgg_acc:.3f}")
print(f"CNN Model Accuracy: {cnn_acc:.3f}")

63/63 [==============================] - 141s 2s/step - loss: 2.2558 - accuracy: 0.9745
63/63 [==============================] - 8s 123ms/step - loss: 0.4449 - accuracy: 0.7965
VGG Model Accuracy: 0.975
CNN Model Accuracy: 0.797

The models' performance differed significantly, according to the final review. The refined VGG model's remarkable 97.5% accuracy rate shows how well it uses pre-trained features to provide accurate categorisation. Comparing the bespoke CNN with augmentation to the VGG model, the latter showed difficulties in feature extraction and generalisation, as seen by its lower accuracy of 79.7%.

# Plot confusion matrix
def plot_confusion_matrix(cm, labels):
    plt.figure(figsize=(8, 6))
    plt.imshow(cm, interpolation="nearest", cmap="Blues")
    plt.title("Confusion Matrix")
    plt.colorbar()
    tick_marks = np.arange(len(labels))
    plt.xticks(tick_marks, labels, rotation=45)
    plt.yticks(tick_marks, labels)

    # Normalize the confusion matrix
    cm_normalized = cm.astype("float") / cm.sum(axis=1)[:, np.newaxis]
    for i, j in np.ndindex(cm.shape):
        plt.text(j, i, f"{cm[i, j]} ({cm_normalized[i, j]:.2f})",
                 horizontalalignment="center",
                 color="white" if cm[i, j] > cm.max() / 2 else "black")

    plt.tight_layout()
    plt.ylabel("True label")
    plt.xlabel("Predicted label")
    plt.show()

# Confusion matrices for both models
labels = ["Cat", "Dog"]
vgg_cm = confusion_matrix(test_labels, vgg_preds)
cnn_cm = confusion_matrix(test_labels, cnn_preds)

print("VGG Confusion Matrix:")
plot_confusion_matrix(vgg_cm, labels)

print("CNN Confusion Matrix:")
plot_confusion_matrix(cnn_cm, labels)

VGG Confusion Matrix:

CNN Confusion Matrix:

The substantial performance difference between the customised CNN and the optimised VGG model is demonstrated by the confusion matrices. With only 27 cats and 24 dogs misclassified, the VGG model has a 97.5% accuracy rate, correctly categorising 98% of dogs and 97% of cats. CNN, on the other hand, misclassified 221 dogs as cats and 186 cats as dogs, with a 79.7% accuracy rate.

# Classification reports
vgg_report = classification_report(test_labels, vgg_preds, target_names=["Cat", "Dog"])
cnn_report = classification_report(test_labels, cnn_preds, target_names=["Cat", "Dog"])

print("VGG Classification Report:\n", vgg_report)
print("CNN Classification Report:\n", cnn_report)

VGG Classification Report:
               precision    recall  f1-score   support

         Cat       0.98      0.97      0.97      1000
         Dog       0.97      0.98      0.97      1000

    accuracy                           0.97      2000
   macro avg       0.97      0.97      0.97      2000
weighted avg       0.97      0.97      0.97      2000

CNN Classification Report:
               precision    recall  f1-score   support

         Cat       0.79      0.81      0.80      1000
         Dog       0.81      0.78      0.79      1000

    accuracy                           0.80      2000
   macro avg       0.80      0.80      0.80      2000
weighted avg       0.80      0.80      0.80      2000

The classification results demonstrate the improved performance of the fine-tuned VGG model, which achieved 97% accuracy for both cats and dogs with a precision, recall, and F1-score of 0.97. This illustrates how well-rounded and trustworthy its forecasts are. With precision, recall, and F1-scores ranging from 0.79 to 0.81, the custom CNN, in contrast, attained 80% accuracy, suggesting more frequent misclassifications.

# Precision-Recall Curve
vgg_precision, vgg_recall, _ = precision_recall_curve(test_labels, vgg_probs)
cnn_precision, cnn_recall, _ = precision_recall_curve(test_labels, cnn_probs)

# Average Precision Scores
vgg_ap = average_precision_score(test_labels, vgg_probs)
cnn_ap = average_precision_score(test_labels, cnn_probs)

plt.figure(figsize=(8, 6))
plt.plot(vgg_recall, vgg_precision, label=f"VGG (AP={vgg_ap:.2f})")
plt.plot(cnn_recall, cnn_precision, label=f"CNN (AP={cnn_ap:.2f})")
plt.title("Precision-Recall Curve")
plt.xlabel("Recall")
plt.ylabel("Precision")
plt.legend()
plt.grid(True)
plt.show()

The improved performance of the tweaked VGG model over the bespoke CNN is demonstrated by the precision-recall curve. With an average precision (AP) of 0.97, the VGG model demonstrated a great capacity to reduce false positives and negatives while maintaining high precision across all recall levels. CNN, on the other hand, got an AP of 0.88, indicating weaker generalisation and a steeper drop in precision as recall rises. The CNN's unpredictability highlights the drawbacks of learning from scratch without using pre-trained features, whereas the VGG's near-flat curve highlights its resilience.

# Identify misclassified examples
vgg_misclassified = np.where(vgg_preds != test_labels)[0]
cnn_misclassified = np.where(cnn_preds != test_labels)[0]

# Plot misclassified examples for VGG
print("Misclassified Examples by VGG:")
for idx in vgg_misclassified[:5]:  # Display up to 5 examples
    img = test_images[idx]
    label = test_labels[idx]
    plt.imshow(img.astype("uint8"))
    plt.title(f"True: {'Dog' if label else 'Cat'}, Pred: {'Dog' if vgg_preds[idx] else 'Cat'}")
    plt.axis("off")
    plt.show()

# Plot misclassified examples for CNN
print("Misclassified Examples by CNN:")
for idx in cnn_misclassified[:5]:  # Display up to 5 examples
    img = test_images[idx]
    label = test_labels[idx]
    plt.imshow(img.astype("uint8"))
    plt.title(f"True: {'Dog' if label else 'Cat'}, Pred: {'Dog' if cnn_preds[idx] else 'Cat'}")
    plt.axis("off")
    plt.show()

Misclassified Examples by VGG:

Misclassified Examples by CNN:

Conclusion

The assessment unequivocally shows that the optimised VGG model outperforms the customised CNN. With a balanced confusion matrix, superior precision-recall metrics (AP=0.97), and a test accuracy of 97.5%, the VGG model successfully uses transfer learning to produce strong and trustworthy predictions. The custom CNN's shortcomings in feature extraction and generalisation, however, were evident in its lower accuracy of 79.7% with an AP of 0.88, even with augmentation techniques.